package com.yanggu.spark.core.accumulator

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object UseWordCountAccumulator {

  def main(args: Array[String]): Unit = {
    //获取环境
    val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("WordCount")

    //获取上下文
    val sc: SparkContext = new SparkContext(conf)

    //创建累加器
    val accumulator = new WordCountAccumulator
    //在sparkContext中注册累加器
    sc.register(accumulator, classOf[WordCountAccumulator].getSimpleName)

    //读取每一行
    val lines: RDD[String] = sc.textFile("input/word.txt")

    //对rdd的数据进行循环遍历
    lines.foreach(_.split(" ").foreach(accumulator.add))

    //读取累加器中的值
    accumulator.value.toList.sortBy(-_._2) foreach println

    sc.stop()
  }

}
